#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
__author__ = 'AJay'
__mtime__ = '2019/3/26 0026'

"""
import datetime
import os
import random
import re
import time

from concurrent.futures import ThreadPoolExecutor, as_completed

import pymysql
import requests
from pyquery import PyQuery as pq
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.wait import WebDriverWait

chrome_options = webdriver.ChromeOptions()
chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
chrome_options.add_argument('--headless')


def get_proxy():
    return requests.get("http://127.0.0.1:5010/get?type=https").json()


def index_page(page):
    """
    抓取索引页
    :param page: 页码
    """
    print('正在爬取第', page, '页')
    retry_count = 5
    proxy = get_proxy().get("proxy")
    while retry_count > 0:
        try:
            html = requests.get('https://avmoo.casa/cn/released/page/' + str(page),
                                proxies={"https": "http://{}".format(proxy)})
            if html.status_code == 200:
                # 使用代理访问
                get_list(html.text)
                html.close()
                retry_count = 0
        except Exception:
            retry_count -= 1


def get_list(html):
    doc = pq(html)
    items = doc('.movie-box').items()
    for item in items:
        id_c = item.find("date:eq(0)").text()
        data_count = query(id_c)
        if data_count == 0:
            text = item.attr("href")
            get_detail_url(text)


def get_detail_url(detail_url):
    start_time = time.perf_counter()
    r_count = 5
    proxy = get_proxy().get("proxy")
    while r_count > 0:
        try:
            html = requests.get('https:' + detail_url, proxies={"https": "http://{}".format(proxy)})
            # 使用代理访问
            if html.status_code == 200:
                get_detail(html.text, start_time)
                html.close()
                r_count = 0
        except Exception:
            r_count -= 1


def get_detail(html, start_time):
    doc = pq(html)
    title = doc(".container h3").text()
    id_code = doc(".col-md-3 p:eq(0) span:eq(1)").text()
    release_date = doc(".col-md-3 p:eq(1)").text().replace('发行时间: ', '')
    length = doc(".col-md-3 p:eq(2)").text().replace('长度: ', '')
    director = ''
    manufacturer = ''
    publisher = ''
    movie_type = ''
    series = ''
    for i in range(3, 15):
        if doc(".col-md-3 p:eq(" + str(i) + ")").text().find("导演") >= 0:
            director = doc(".col-md-3 p:eq(" + str(i) + ") a").text()
        if doc(".col-md-3 p:eq(" + str(i) + ")").text().find("制作商") >= 0:
            manufacturer = doc(".col-md-3 p:eq(" + str(i + 1) + ") a").text()
        if doc(".col-md-3 p:eq(" + str(i) + ")").text().find("发行商") >= 0:
            publisher = doc(".col-md-3 p:eq(" + str(i + 1) + ") a").text()
        if doc(".col-md-3 p:eq(" + str(i) + ")").text().find("系列") >= 0:
            series = doc(".col-md-3 p:eq(" + str(i + 1) + ") a").text()
        if doc(".col-md-3 p:eq(" + str(i) + ")").text().find("类别") >= 0:
            its = doc(".col-md-3 p:eq(" + str(i + 1) + ") span").items()
            for it in its:
                movie_type += ',' + it.text()
            if len(movie_type) > 2:
                movie_type = movie_type[1:]

    actor = ''
    its = doc("#avatar-waterfall").items()
    for it in its:
        actor += ',' + it.find("span").text()
    if len(actor) > 2:
        actor = actor[1:]

    # 下载图片
    path = mkdir(title, series)
    download_img(path, doc(".bigImage").attr('href'))
    its = doc("#sample-waterfall a").items()
    for it in its:
        download_img(path, it.attr('href'))

    # 使用 connect 方法，传入数据库地址，账号密码，数据库名就可以得到你的数据库对象
    db = pymysql.connect(host="localhost", user="root", password="ziyeibin", db="moo")

    # 接着我们获取 cursor 来操作我们的 avIdol 这个数据库
    cursor = db.cursor()

    # 插入一条记录
    sql = "insert into moo_movie(id_code, title,release_date,length,director,manufacturer,publisher,series,movie_type,actor,create_time) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
    params = (id_code, title, release_date, length, director, manufacturer, publisher, series, movie_type, actor,
              datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

    cursor.execute(sql, params)
    db.commit()
    # 最后我们关闭这个数据库的连接
    db.close()
    stop_time = time.perf_counter()
    print('获取内容:' + title + '--耗时：' + str(stop_time - start_time))


#  html = browser.get('https:'+detail_url)
#   doc = pq(html)

def download_img(path, img_url):
    img_name = os.path.basename(img_url)
    print(path + '/' + img_name)
    header = {}  # 设置http header，视情况加需要的条目，这里的token是用来鉴权的一种方式
    r_count = 5
    proxy = get_proxy().get("proxy")
    while r_count > 0:
        try:
            html = requests.get(img_url, headers=header, stream=True, proxies={"https": "http://{}".format(proxy)})
            # 使用代理访问
            if html.status_code == 200:
                open(path + '/' + img_name, 'wb').write(html.content)  # 将内容写入图片
                html.close()
                r_count = 0
        except Exception:
            r_count -= 1


def mkdir(folder_name, series):
    zm_cop = re.compile("[^a-z^A-Z]")  # 匹配不是中文、大小写、数字的其他字符
    second_folder = zm_cop.sub('', folder_name)  # 将string1中匹配到的字符替换成空字符
    path = './temp'
    # 先创建带编号文件夹
    if not os.path.exists(path + '/' + second_folder):
        path = path + '/' + second_folder
        os.mkdir(path)
    else:
        path = path + '/' + second_folder
    # 存在系列创建带系列文件夹
    if series != '':
        if not os.path.exists(path + '/' + series):
            path = path + '/' + series
            os.mkdir(path)
        else:
            path = path + '/' + series
    # 创建带编号文件夹
    if not os.path.exists(path + '/' + folder_name):
        path = path + '/' + folder_name
        os.mkdir(path)
    else:
        path = path + '/' + folder_name
    print(path)
    return path


def query(id_code):
    db = pymysql.connect(host="localhost", user="root", password="ziyeibin", db="moo")
    cursor = db.cursor()
    # 插入一条记录
    sql = "select count(1) from moo_movie where id_code= %s"
    param = id_code
    cursor.execute(sql, param)
    result = cursor.fetchone()
    # 最后我们关闭这个数据库的连接
    db.close()
    return result[0]


def main():
    # for page in range(1, 10000):
    #     index_page(page)

    with ThreadPoolExecutor(max_workers=10) as t:
        #    obj_list = []
        for page in range(1, 11):
            t.submit(index_page, page)


#         obj_list.append(obj)

# for future in as_completed(obj_list):
#     data = future.result()
#     print(f"main: {data}")


def tests(count):
    sleeptime = random.randint(0, 5)
    time.sleep(sleeptime)
    print(count)


if __name__ == '__main__':
    if not os.path.exists('temp'):
        os.mkdir('temp')
    main()

    # str1 = 'PARATHD-3260 水电费水电费水电费颠倒是非'
    # str_list = str1.split(" ")
    # print(str_list[0])
